/*************************************************************************
    > File Name    : DictProducer.cc
    > Author       : LiuLei
    > Mail         : 2261459486@qq.com 
    > Created Time : 2022年11月29日 星期二 10时26分22秒
 ************************************************************************/

#include "../../include/DictBuilder/DictProducer.h"
#include "../../include/DictBuilder/SplitTool.h"
#include <sys/types.h>
#include <dirent.h>
#include <string.h>
#include <ctype.h>

#include <iostream>
#include <fstream>
#include <sstream>

using std::cout;
using std::endl;
using std::ifstream;
using std::istringstream;
using std::ofstream;

DictProducer::DictProducer(Configuration &conf, SplitTool *splitTool, int flag)
: _conf(conf)
, _splitTool(splitTool)
, _flag(flag)
{
    // 1、加载配置文件指示的语料库目录下的所有文件
    unordered_map<string, string> &confMap = _conf.getConfigMap();
    
    if(0 == flag)   // 英文
    {
        string dir = confMap["corpus-en"];
        this->getFiles(dir, _files);

        // 2、创建词典
        cout << "Creating English dictionary, please wait..." << endl;
        this->buildEnDict();

        // 3、保存词典文件
        string filepath = confMap["dict-en"];
        cout << "Saving English dictionary, please wait..." << endl;
        this->storeDict(filepath);

        // 4、创建索引
        cout << "Creating English index, please wait..." << endl;
        this->buildIndex(filepath);
        
        // 5、保存索引文件
        cout << "Saving English index, please wait..." << endl;
        filepath = confMap["index-en"];
        storeIndex(filepath);
    }
    else            // 中文
    {
        string dir = confMap["corpus-cn"];
        this->getFiles(dir, _files);

        // 2、创建词典
        cout << "Creating Chinese dictionary, please wait..." << endl;
        this->buildCnDict();

        // 3、保存词典文件
        string filepath = confMap["dict-cn"];
        cout << "Saving Chinese dictionary, please wait..." << endl;
        this->storeDict(filepath);

        // 4、创建索引
        cout << "Creating Chinese index, please wait..." << endl;
        this->buildIndex(filepath);
        
        // 5、保存索引文件
        cout << "Saving Chinese index, please wait..." << endl;
        filepath = confMap["index-cn"];
        storeIndex(filepath);
    }
}

DictProducer::~DictProducer()
{
    if(_splitTool)
    {
        delete _splitTool;
        _splitTool = nullptr;
    }
}

void DictProducer::buildEnDict()
{
    // 通过配置类获取stopWordList
    unordered_set<string> stopWordList = _conf.getStopWorlList();

    for(auto &file : _files)
    {
        ifstream ifs;
        ifs.open(file);
        if(!ifs.good())
        {
            cout << "Open " << file << " error in DictProducer::buildEnDict()!" << endl;
            return ;
        }

        string text;
        // 对文件进行逐行读取
        while(getline(ifs, text))
        {
            istringstream iss(text);
            string word;
            while(iss >> word)
            {
                // 对单词进行转换，转换为小写并过滤标点
                word = this->cleanUpWord(word);
                
                // 过滤空字符串
                if(word.length() != 0)
                {
                    // 过滤stopWord
                    unordered_set<string>::iterator stop = stopWordList.find(word);
                    if(stop != stopWordList.end())
                    {
                        // 命中
                        continue;
                    }
                    else
                    {
                        ++_dict[word];
                    }
                }
            }
        }
        ifs.close();
    }
}

void DictProducer::buildCnDict()
{
    // 获取stopWordList
    unordered_set<string> stopWordList = _conf.getStopWorlList();

    for(auto &file : _files)
    {
        // 打开文件并将ptr指向文件末尾
        ifstream ifs(file, std::ios::ate);
        if(!ifs.good())
        {
            cout << "Open " << file << " error in DictProducer::buildCnDict()!" << endl;
            return ;
        }
        
        // 一次性读取一篇文章
        size_t length = ifs.tellg();
        ifs.seekg(std::ios::beg);
        char *buff = new char[length + 1];
        ifs.read(buff, length + 1);
        string article(buff);
        //cout << "The article's size : " << article.size() << endl;
        
        vector<string> results = _splitTool->cut(article);

        for(auto &word : results)
        {
            // 过滤空字符串
            if(word.length() != 0)
            {
                // 过滤stopWord
                unordered_set<string>::iterator stop = stopWordList.find(word);
                if(stop == stopWordList.end() && this->getUtf8ByteNum(word[0] == 3))
                {
                    ++_dict[word];
                }
            }
        }
        ifs.close();
        delete [] buff;
    }
}

void DictProducer::buildIndex(const string &filepath)
{
    // 先加载词典文件中的数据到词典vector中
    ifstream ifs;
    ifs.open(filepath);
    if(!ifs.good())
    {
        cout << "Open " << filepath << " error in DictProducer::buildIndex()!" << endl;
        return ;
    }

    string line, word;
    int i = 0;      // 记录的是词典vector的下标
    while(getline(ifs, line))
    {
        istringstream iss(line);
        iss >> word;

        size_t charNums = word.size() / this->getUtf8ByteNum(word[0]);
        for(size_t idx = 0, n = 0; n != charNums; ++idx, ++n)
        {
            size_t charLen = this->getUtf8ByteNum(word[idx]);
            string subWord = word.substr(idx, charLen);
            _index[subWord].insert(i);
            idx += (charLen - 1);
        }
        ++i;
    }
}

void DictProducer::storeDict(const string &filepath)
{
    ofstream ofs;
    ofs.open(filepath, std::ios::out | std::ios::trunc);
    if(!ofs.good())
    {
        cout << "Open " << filepath << " error in DictProducer::storeDict()!" << endl;
        return ;
    }

    for(auto &elem : _dict)
    {
        ofs << elem.first << "  " << elem.second << endl;
    }
    ofs.close();
}

void DictProducer::storeIndex(const string &filepath)
{
    ofstream ofs;
    ofs.open(filepath, std::ios::out | std::ios::trunc);
    if(!ofs.good())
    {
        cout << "Open " << filepath << " erro in DictProducer::storeIndex()!" << endl;
        return ;
    }

    for(auto &elem : _index)
    {
        ofs << elem.first << "  ";
        for(auto &it : elem.second)
        {
            ofs << it << " ";
        }
        ofs << endl;
    }
    ofs.close();
}

void DictProducer::showFiles() const
{
    cout << "语料库文件的绝对路径集合 :" << endl;
    for(auto &it : _files)
    {
        cout << it << endl;
    }
}

void DictProducer::showDict() const
{
    cout << "词典 :" << endl;
    for(auto &it : _dict)
    {
        cout << it.first << "  " << it.second << endl;
    }
    cout << "词典大小 : " << _dict.size() << endl;
}

void DictProducer::getFiles(string path, vector<string> &files)
{
    DIR *dir;
    struct dirent *ptr;
    
    if(nullptr == (dir = ::opendir(path.c_str())))
    {
        cout << "Open directory " << path << " error in DictProducer::getFiles()!" << endl;
        return ;
    }

    while((ptr = ::readdir(dir)) != nullptr)
    {
        if(::strcmp(ptr->d_name, ".") == 0 || ::strcmp(ptr->d_name, "..") == 0)
        {
            continue;
        }
        // 文件
        else if(ptr->d_type == 8)
        {
            files.push_back(path + ptr->d_name);
        }
        // 目录
        else if(ptr->d_type == 4)
        {
            this->getFiles(path + ptr->d_name + "/", files);
        }
    }
    ::closedir(dir);
}

string DictProducer::cleanUpWord(const string &word)
{
    string ret;
    for(auto it = word.begin(); it != word.end(); ++it)
    {
        if(::isalpha(*it))
        {
            ret += tolower(*it);
        }
    }
    return ret;
}

size_t DictProducer::getUtf8ByteNum(const char byte)
{
    int byteNum = 0;
    for(size_t i = 0; i < 6; ++i)
    {
        if(byte & (1 << (7 - i)))
        {
            ++byteNum;
        }
        else
        {
            break;
        }
    }
    return byteNum == 0 ? 1 : byteNum;
}
